from bs4 import BeautifulSoup
import requests

def my_request(method, url, **kwargs):
    """
    封装请求接口的函数
    :param url: 请求的url
    :param kwargs:
    :return:
    """
    rep = requests.request(method=method, url=url, **kwargs)
    rep.encoding = 'utf-8'
    return rep.text

def my_get_passage_txt(html_str):
    text_soup = BeautifulSoup(html_str, 'lxml')
    value_tag = text_soup.find(name='div', id="content")
    passage_text_soup = BeautifulSoup(value_tag.prettify(), 'lxml')
    return passage_text_soup.find('div').text

def main():
    host = "https://www.ibiquges.com"
    url = f"{host}/7/7014/?dd=dd&t=0.5731896837840009"
    f = open('.\shenmu.txt', 'w', encoding='utf-8')
    # 获取章节页面html
    html_page = my_request(method="get", url=url)
    # print(type(html_page))
    # print(html_page)
    # 初始化整个页面为一个可解析的对象
    soup = BeautifulSoup(html_page, 'lxml')
    # 获取章节目录所在的节点
    passage_tag = soup.find(name='dl')
    # 获取所有的章节节点
    tags = passage_tag.find_all(name='a')
    for tag in tags:
        passage_name = tag.text
        print("章节：{}".format(passage_name))
        f.write(passage_name)
        passage_url = "{}{}".format(host, tag.attrs.get("href"))
        print("章节url：{}".format(passage_url))
        html = my_request('get', passage_url)
        value = my_get_passage_txt(html)
        f.write(value)
        index = tags.index(tag)+1
        print("当前进度为：{}%".format(round(index*100/len(tags), 2)))
    f.close()

if __name__ == '__main__':
    main()
